﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;

namespace SimpleCrawler
{
    class SimpleCrawler
    {
        private Hashtable urls = new Hashtable();
        private int count = 0;
        private string hostUrl;
        private string originalUrl;
        public SimpleCrawler(string originalUrl,string hostUrl)
        {
            this.hostUrl = hostUrl;
            this.originalUrl = originalUrl;
            this.urls.Add(originalUrl, false);
        }
        private bool isWebpage(string url)
        {
            string[] fileTypes = { "/", ".html", ".htm", ".aspx", ".php", ".jsp" };
            foreach (string fileType in fileTypes)
            {
                if (url.EndsWith(fileType)) return true;
            }
            return false;
        }
        static void Main(string[] args)
        {
            SimpleCrawler myCrawler = new SimpleCrawler("http://www.cnblogs.com/Rorschach-XR/", "http://www.cnblogs.com/Rorschach-XR/");

            new Thread(myCrawler.Crawl).Start();
        }

        private void Crawl()
        {
            Console.WriteLine("开始爬行了.... ");
            while (true)
            {
                string current = null;
                foreach (string url in urls.Keys)
                {
                    if ((bool)urls[url]) continue;
                    if(isWebpage(url))
                    {
                        current = url;
                        break;
                    }
                    
                }

                if (current == null || count > 10) break;
                Console.WriteLine("爬行" + current + "页面!");
                string html = DownLoad(current); // 下载
                urls[current] = true;
                count++;
                Parse(html);//解析,并加入新的链接
                Console.WriteLine("爬行结束");
            }
            //while (true) ;
        }

        public string DownLoad(string url)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);
                string fileName = count.ToString();
                File.WriteAllText(fileName, html, Encoding.UTF8);
                return html;
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                return "";
            }
        }

        private void Parse(string html)
        {
            string strRef = @"(href|HREF)[]*=[]*[""'][^""'#>]+[""']";
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1)
                          .Trim('"', '\"', '#', '>');
                if (strRef.Length == 0) continue;
                if (strRef.StartsWith("/")) strRef += hostUrl;
                if (urls[strRef] == null) urls[strRef] = false;
            }
        }
    }
}
